package main

import (
	"encoding/csv"
	"fmt"
	"github.com/xuri/excelize/v2"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	filePath   = "./Reptile/records.xls"
	filePathV2 = "./Reptile/records.xlsx"
	requestUrl = "https://www.aquanliang.com/blog/page/"
)

func getTitle(result string) (title string) {
	re := regexp.MustCompile(`<div class="_3_JaaUmGUCjKZIdiLhqtfr">(?s:(.*?))</div>`)
	if re == nil {
		fmt.Println("regexp.MustCompile err")
		return ""
	}

	// 取关键信息
	res := re.FindAllString(result, -1)
	// 替换多余项
	title = strings.Replace(res[0], "</div>", "", -1)
	title = strings.Replace(title, "<div class=\"_3_JaaUmGUCjKZIdiLhqtfr\">", "", -1)

	return title
}

func getTime(result string) (time string) {
	re := regexp.MustCompile(`</div>[0-9]+-[0-9]+-[0-9]+</div>`)
	if re == nil {
		fmt.Println("regexp.MustCompile err")
		return ""
	}

	// 取关键信息
	res := re.FindAllString(result, -1)
	// 替换多余项
	time = strings.Replace(res[0], "</div>", "", -1)

	return time
}

func getReadNum(result string) (num string) {
	re := regexp.MustCompile(`</div>[0-9]+</div>`)
	if re == nil {
		fmt.Println("regexp.MustCompile err")
		return ""
	}

	// 取关键信息
	res := re.FindAllString(result, -1)
	// 替换多余项
	num = strings.Replace(res[0], "</div>", "", -1)

	return num
}

func getCoverImage(result string) (image string) {
	re := regexp.MustCompile(`https://quan-1259287960.cos.ap-guangzhou.myqcloud.com/1000029/(?s:(.*?)):cover`)
	if re == nil {
		fmt.Println("regexp.MustCompile err")
		return ""
	}

	// 取关键信息
	res := re.FindAllString(result, -1)
	// 替换多余项
	image = strings.Replace(res[0], "\" decoding=\"async\" style=\"position:absolute;top:0;left:0;bottom:0;right:0;box-sizing:border-box;padding:0;border:none;margin:auto;display:block;width:0;height:0;min-width:100%;max-width:100%;min-height:100%;max-height:100%;object-fit:cover", "", -1)

	return image
}

// HttpGet 发送get请求
func HttpGet(url string) (result string, err error) {
	resp, err1 := http.Get(url) //发送get请求
	if err1 != nil {
		err = err1
		return
	}
	defer resp.Body.Close()

	// 读取网页内容
	buf := make([]byte, 1024*4)
	for {
		n, _ := resp.Body.Read(buf)
		if n == 0 {
			break
		}
		// 累加读取的内容
		result += string(buf[:n])
	}
	return
}

// ContentToFile 把内容写入到excel文件
func ContentToFile(fileTitle, fileTime, fileRead, fileImage []string) {
	//创建excel
	f, err := os.Create(filePath)
	if err != nil {
		fmt.Println("os.Create err = ", err)
		return
	}
	// 最后关闭文件
	defer f.Close()
	// 设置表头
	f.WriteString("\xEF\xBB\xBF") // 写入UTF-8 BOM
	w := csv.NewWriter(f)
	w.Write([]string{"标题", "时间", "阅读量", "封面图"})

	n := len(fileRead)
	for j := 0; j < n; j++ {
		// 写内容
		w.Write([]string{fileTitle[j], fileTime[j], fileRead[j], fileImage[j]})
	}
	w.Flush()
}

// ExcelContentToFile 引用第三方库优雅写入excel
func ExcelContentToFile(fileTitle, fileTime, fileRead, fileImage []string) {
	f := excelize.NewFile() // 设置单元格的值
	// 设置列宽
	f.SetColWidth("Sheet1", "A", "A", 80)
	f.SetColWidth("Sheet1", "B", "B", 20)
	f.SetColWidth("Sheet1", "D", "D", 100)
	// 设置表头行高
	f.SetRowHeight("Sheet1", 1, 20)
	// 这里设置表头
	f.SetCellValue("Sheet1", "A1", "标题")
	f.SetCellValue("Sheet1", "B1", "时间")
	f.SetCellValue("Sheet1", "C1", "阅读数")
	f.SetCellValue("Sheet1", "D1", "封面图")

	// 循环写入数据
	line := 1
	n := len(fileRead)
	for j := 0; j < n; j++ {
		line++
		// 写内容
		f.SetCellValue("Sheet1", fmt.Sprintf("A%d", line), fileTitle[j])
		f.SetCellValue("Sheet1", fmt.Sprintf("B%d", line), fileTime[j])
		f.SetCellValue("Sheet1", fmt.Sprintf("C%d", line), fileRead[j])
		f.SetCellValue("Sheet1", fmt.Sprintf("D%d", line), fileImage[j])
	}

	// 保存文件
	if err := f.SaveAs(filePathV2); err != nil {
		fmt.Println(err)
	}
	fmt.Println("写入完成！")
}

// 定义切片，用于收集每页的所需信息
var fileTitles []string
var fileTimes []string
var fileReadNums []string
var fileCoverImages []string

func SpiderPage(i int, page chan int) {
	// 明确爬取的url
	// https://www.aquanliang.com/blog/page/1 下一页+1
	url := requestUrl + strconv.Itoa(i)
	fmt.Printf("正在爬取第%d个网页：%s\n", i, url)

	// 开始爬取网页的内容
	result, err := HttpGet(url)
	if err != nil {
		fmt.Println("HttpGet err = ", err)
		return
	}
	// 解析获取每页的总文章及内容
	re := regexp.MustCompile(`<div class="_1ySUUwWwmubujD8B44ZDzy">(?s:(.*?))<div class="OCOci4ZmKxaQrjcYLswKa">`)
	if re == nil {
		fmt.Println("regexp.MustCompile err")
		return
	}
	results := re.FindAllString(result, -1)
	//fmt.Println(results)
	//fmt.Println(results[0])
	//fmt.Println(results[0][0])

	// 解析获取每页的总文章及内容
	re1 := regexp.MustCompile(`<div class="_3gcd_TVhABEQqCcXHsrIpT">(?s:(.*?))</span>`)
	if re == nil {
		fmt.Println("regexp.MustCompile err")
		return
	}
	contents := re1.FindAllStringSubmatch(results[0], -1)

	// 循环获取每页每篇文章的内容
	for _, data := range contents {
		// 取一个文章的内容
		content := data[0]
		//fmt.Println(content)
		//逐项解析获取对应值，并存入切片中
		fileTitles = append(fileTitles, getTitle(content))
		fileTimes = append(fileTimes, getTime(content))
		fileReadNums = append(fileReadNums, getReadNum(content))
		fileCoverImages = append(fileCoverImages, getCoverImage(content))
	}

	//if len(fileTitles) == 0 || len(fileTimes) == 0 || len(fileReadNums) == 0 || len(fileCoverImages) == 0 {
	//	fmt.Println("i = ", i)
	//}
	//fmt.Println("fileTitle = ", i, fileTitles)
	//fmt.Println("fileTime = ", i, fileTimes)
	//fmt.Println("fileRead = ", i, fileReadNums)
	wg.Done() // goroutine结束就登记-1
	//fmt.Println("fileImage = ", i, fileCoverImages)

	////把内容写入到excel文件
	//excelStoreJoyToFile(i, fileTitles, fileTimes, fileReadNums, fileCoverImages)
	//StoreJoyToFile(fileTitles, fileTimes, fileReadNums, fileCoverImages)

	// 返回完成页数
	page <- i
}

func DoWork(start, end int) {
	fmt.Printf("正在爬取页数范围是第%d页到第%d页。\n", start, end)
	page := make(chan int)
	for i := start; i <= end; i++ {
		// 爬取主页面
		wg.Add(1)
		go SpiderPage(i, page)
		// 防止爬虫速率过快
		time.Sleep(2 * time.Second)
		// 防止网络延迟请求量过多
		if i%3 == 0 {
			wg.Wait()
		}
	}

	for i := start; i <= end; i++ {
		fmt.Printf("第%d页已经爬取完成。\n", <-page)
		//if i == end {
		//	fmt.Println("开始写入。")
		//	StoreJoyToFile(fileTitles, fileTimes, fileReadNums, fileCoverImages)
		//}
	}
}

// 声明全局等待组变量
var wg sync.WaitGroup

func main() {
	var start, end int
	fmt.Print("请输入起始页(>=1) :")
	fmt.Scan(&start)
	fmt.Print("请输入结束页(>=起始页) :")
	fmt.Scan(&end)
	//把内容写入到excel文件

	//开始工作
	DoWork(start, end)
	//wg.Wait()
	fmt.Println("开始写入excel文件。")
	ExcelContentToFile(fileTitles, fileTimes, fileReadNums, fileCoverImages)
}
